{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RNN for MNIST digits classification\n",
    "\n",
    "In this example, we demostrate MNIST classification using RNN. Although images are not sequential in nature, we can reshape the tensor such that an image is a sequence of pixels.\n",
    "\n",
    "The data preparation is similar to previous examples in MLP and CNN. After building a 1-layer RNN, the training is similar to MLP and CNN. In this example we achieve `~98.2%` test accuracy in `20 epochs`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"SimpleRNN_MNIST\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "simple_rnn_3 (SimpleRNN)     (None, 256)               72960     \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 10)                2570      \n",
      "=================================================================\n",
      "Total params: 75,530\n",
      "Trainable params: 75,530\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/20\n",
      "469/469 [==============================] - 18s 38ms/step - loss: 0.6319 - accuracy: 0.8352\n",
      "Epoch 2/20\n",
      "469/469 [==============================] - 18s 39ms/step - loss: 0.2692 - accuracy: 0.9237\n",
      "Epoch 3/20\n",
      "469/469 [==============================] - 17s 36ms/step - loss: 0.2004 - accuracy: 0.9412\n",
      "Epoch 4/20\n",
      "469/469 [==============================] - 17s 35ms/step - loss: 0.1632 - accuracy: 0.9524\n",
      "Epoch 5/20\n",
      "469/469 [==============================] - 16s 35ms/step - loss: 0.1394 - accuracy: 0.9586\n",
      "Epoch 6/20\n",
      "469/469 [==============================] - 17s 35ms/step - loss: 0.1240 - accuracy: 0.9631\n",
      "Epoch 7/20\n",
      "469/469 [==============================] - 19s 41ms/step - loss: 0.1103 - accuracy: 0.9674\n",
      "Epoch 8/20\n",
      "469/469 [==============================] - 18s 39ms/step - loss: 0.1003 - accuracy: 0.9704\n",
      "Epoch 9/20\n",
      "469/469 [==============================] - 18s 38ms/step - loss: 0.0919 - accuracy: 0.9728\n",
      "Epoch 10/20\n",
      "469/469 [==============================] - 17s 36ms/step - loss: 0.0845 - accuracy: 0.9744\n",
      "Epoch 11/20\n",
      "469/469 [==============================] - 17s 37ms/step - loss: 0.0785 - accuracy: 0.9765\n",
      "Epoch 12/20\n",
      "469/469 [==============================] - 17s 37ms/step - loss: 0.0723 - accuracy: 0.9782\n",
      "Epoch 13/20\n",
      "469/469 [==============================] - 17s 37ms/step - loss: 0.0678 - accuracy: 0.9799\n",
      "Epoch 14/20\n",
      "469/469 [==============================] - 16s 35ms/step - loss: 0.0645 - accuracy: 0.9811\n",
      "Epoch 15/20\n",
      "469/469 [==============================] - 18s 38ms/step - loss: 0.0606 - accuracy: 0.9814\n",
      "Epoch 16/20\n",
      "469/469 [==============================] - 19s 40ms/step - loss: 0.0565 - accuracy: 0.9832\n",
      "Epoch 17/20\n",
      "469/469 [==============================] - 16s 34ms/step - loss: 0.0531 - accuracy: 0.9846\n",
      "Epoch 18/20\n",
      "469/469 [==============================] - 17s 37ms/step - loss: 0.0510 - accuracy: 0.9847\n",
      "Epoch 19/20\n",
      "469/469 [==============================] - 17s 36ms/step - loss: 0.0481 - accuracy: 0.9858\n",
      "Epoch 20/20\n",
      "469/469 [==============================] - 18s 38ms/step - loss: 0.0464 - accuracy: 0.9861\n",
      "79/79 [==============================] - 1s 12ms/step - loss: 0.0582 - accuracy: 0.9815\n",
      "\n",
      "Test accuracy: 98.2%\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import numpy as np\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, SimpleRNN\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.datasets import mnist\n",
    "\n",
    "# load mnist dataset\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "\n",
    "# compute the number of labels\n",
    "num_labels = len(np.unique(y_train))\n",
    "\n",
    "# convert to one-hot vector\n",
    "y_train = to_categorical(y_train)\n",
    "y_test = to_categorical(y_test)\n",
    "\n",
    "# resize and normalize\n",
    "image_size = x_train.shape[1]\n",
    "x_train = np.reshape(x_train,[-1, image_size, image_size])\n",
    "x_test = np.reshape(x_test,[-1, image_size, image_size])\n",
    "x_train = x_train.astype('float32') / 255\n",
    "x_test = x_test.astype('float32') / 255\n",
    "\n",
    "# network parameters\n",
    "input_shape = (image_size, image_size)\n",
    "batch_size = 128\n",
    "units = 256\n",
    "\n",
    "# model is RNN with 256 units, input is 28-dim vector 28 timesteps\n",
    "model = Sequential(name='SimpleRNN_MNIST')\n",
    "model.add(SimpleRNN(units=units,\n",
    "                    input_shape=input_shape))\n",
    "model.add(Dense(num_labels, activation='softmax'))\n",
    "model.summary()\n",
    "\n",
    "# loss function for one-hot vector\n",
    "# use of sgd optimizer\n",
    "# accuracy is good metric for classification tasks\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='sgd',\n",
    "              metrics=['accuracy'])\n",
    "# train the network\n",
    "model.fit(x_train, y_train, epochs=20, batch_size=batch_size)\n",
    "\n",
    "loss, acc = model.evaluate(x_test, y_test, batch_size=batch_size)\n",
    "print(\"\\nTest accuracy: %.1f%%\" % (100.0 * acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
